import datetime
import pandas as pd
import numpy as np
import Analysis.General as General
import statsmodels.api as sm
import Core.Gadget as Gadget

# ---df里为'nan'的用0填充---
def Fillna(df):
    #
    columns = df.columns.values
    for c in range(len(df.columns)):
        eachcolumn = df.iloc[:, c:c + 1].values
        newcolumn = []
        for i in range(len(eachcolumn)):
            if str(eachcolumn[i][0]) != 'nan':
                newcolumn.append(eachcolumn[i][0])
            else:
                newcolumn.append(0)
        df[columns[c]] = newcolumn

# ---在SQL中表格填入---
def BuildTable(database, newFactorName, df):
    #
    newDocuments = []

    for i in range(len(df)):
        dfsingle = df.iloc[i:i+1,:]
        newDocument = {}
        newDocument["Symbol"] = dfsingle['Symbol'].values[0]
        newDocument["Value"] = float(dfsingle['Value'].values[0])
        newDocument["Period"] = int(dfsingle['Period'].values[0])

        datetime1 = dfsingle["DateTime"].values[0]
        if isinstance(datetime1, np.datetime64):
            datetime1 = (datetime1 - np.datetime64('1970-01-01T00:00:00Z')) / np.timedelta64(1, 's')
            datetime1 = datetime.datetime.utcfromtimestamp(datetime1)

        newDocument["DateTime"] = datetime1
        newDocument["Date"] = datetime1

        newDocument["Key2"] = newDocument["Symbol"] + "_" + datetime.datetime.strftime(datetime1,'%Y-%m-%d')
        newDocuments.append(newDocument)

    database.Upsert_Many('factor', newFactorName, [], newDocuments)

# ---将Y、X根据symbol对齐---
def DataAlign(dfy, dfx):
    #
    len1 = len(dfy.columns)
    df = pd.merge(dfy, dfx, how='outer', on='Symbol')
    df.dropna(inplace=True)

    y = df.iloc[:,:len1]
    x = df.iloc[:,len1:]
    #
    return y,x,df

def Get_Industry_Exposure(symbols, datetime2):
    # 找到最近的一次 Release Date
    releaseDate = Gadget.FindReleaseDate(datetime2)
    #
    filter = {"DateTime": releaseDate, "Type": "Citic"}
    documents = database.Find("Stock", "Industry", filter)
    dfSymbolwithIndustry = Gadget.DocumentsToDataFrame(documents, keep=["DateTime", "Symbol", "Industry"])

    #symbols 和 df 可以Merge 一下
    #dfSymbolwithIndustry = dfSymbolwithIndustry[dfSymbolwithIndustry['Symbol'].isin(symbols)]  #

    #行业虚拟变量
    dummy_ranks = pd.get_dummies(dfSymbolwithIndustry['Industry'], prefix=None)

    dfSymbolwithIndustry = pd.concat([dfSymbolwithIndustry.iloc[:, :2], dummy_ranks],axis=1)

    return dfSymbolwithIndustry

def Neutralize(database, factorName, newFactorName, reportDate, betaFactors=["LnCap"], industry=False):
    #
    # 读取因子-Y
    filter = {"ReportDate": reportDate}
    documents = database.Find("Factor", factorName, filter)
    dfYFactor = Gadget.DocumentsToDataFrame(documents, keep=["Value", "DateTime", "Symbol", "ReportDate", "Period"])

    # 读取因子-Xs （Beta Factor）
    dfX = pd.DataFrame()
    for i in range(len(betaFactors)):
        dfBeta = General.Profile(database, reportDate, betaFactors, instruments=None)
        dfX = pd.concat([dfX, dfBeta], axis=1)

    # 行业虚拟变量
    if industry:
        dfSymbolIndustry = Get_Industry_Exposure(list(dfYFactor["Symbol"]), reportDate)
        dfX = pd.merge(dfX, dfSymbolIndustry, how='outer', on='Symbol')
        Fillna(dfX)

    dfX.drop(['DateTime'],axis=1, inplace=True)
    dfX['Intercept'] = 1

    dfYFactor, dfX, dftotal = DataAlign(dfYFactor, dfX)

    y = dfYFactor['Value']
    x = dfX.iloc[:,:]

    # 建立回归模型
    result = sm.OLS(y, x).fit()

    # 取残差
    resid = result.resid

    #print(result.summary())

    #建新dataframe
    dfFinal = dfYFactor.copy(deep=True)
    dfFinal['Value'] = resid
    dfFinal.columns = ['DateTime','ReportDate','Symbol','Value','Period']

    # 将新因子存到一个新的Table
    BuildTable(database, newFactorName, dfFinal)

    #打印以便校对
    # dftotal['Value_neutral'] = resid
    # dftotal.to_csv('factor_neutralized.csv', encoding='gbk')

    return dfFinal

if __name__ == '__main__':
    #
    from Core.Config import *
    cfgPathFilename = os.getcwd() + "/../config.json"
    config = Config(cfgPathFilename)
    database = config.DataBase("MySQL")

    reportDate = datetime.datetime(2018, 3, 31)
    factorName = "Roe_NetIncome2_TTM"

    df = Neutralize(database, factorName, factorName + "_Neutral", reportDate, betaFactors=["LnCap"], industry=True)


